{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.2"
    },
    "colab": {
      "name": "Bert.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c2f7806b750e4d3cbc7eca56b433e297": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_552d3c4b5cc3492781115be450e4e3b7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f6a30eb703924f39a2d1871cd5d790fa",
              "IPY_MODEL_53daecb4f2d643ecb9dc87ba7b195318"
            ]
          }
        },
        "552d3c4b5cc3492781115be450e4e3b7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f6a30eb703924f39a2d1871cd5d790fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0e993b716be84e71806c8b0028522b2f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 313,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 313,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1d103ef8e9a14ca5b1f57079372a9615"
          }
        },
        "53daecb4f2d643ecb9dc87ba7b195318": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_aa3e0cd9d7d142ab976bc96c56ca295c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 313/313 [00:00&lt;00:00, 780B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f568e4e282254f98a46484d6cbf7e7f9"
          }
        },
        "0e993b716be84e71806c8b0028522b2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1d103ef8e9a14ca5b1f57079372a9615": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aa3e0cd9d7d142ab976bc96c56ca295c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f568e4e282254f98a46484d6cbf7e7f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "36f263d60c1a4071a1ba3a4d3c156d47": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a935ca717870498cb1a524199005ce49",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_142da5ae7175430f866e45c65f530499",
              "IPY_MODEL_ba51cb0320d248b6b3213c00def9c2da"
            ]
          }
        },
        "a935ca717870498cb1a524199005ce49": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "142da5ae7175430f866e45c65f530499": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5c79625a65164c6ab0b95a3f6eec2657",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 81,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 81,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_704bf69a61354623851a86ac08214c6e"
          }
        },
        "ba51cb0320d248b6b3213c00def9c2da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_671076a72248430d9829a73d1f282d7f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 81.0/81.0 [00:25&lt;00:00, 3.23B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0e46c95a421149c3890d08ac8370d6b2"
          }
        },
        "5c79625a65164c6ab0b95a3f6eec2657": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "704bf69a61354623851a86ac08214c6e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "671076a72248430d9829a73d1f282d7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0e46c95a421149c3890d08ac8370d6b2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "06f451a806284f0d90aeea93e347d009": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f3ce0768f9aa4ba3b5a1f6a25353c804",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_bee98a37856642cd9b5e3c29672861c0",
              "IPY_MODEL_91105b770312483bae5338aab57359ed"
            ]
          }
        },
        "f3ce0768f9aa4ba3b5a1f6a25353c804": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bee98a37856642cd9b5e3c29672861c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c4f2791a4fd84ca7a014c946349611d6",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1684058277,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1684058277,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_82c90ab4c7d649e097523d2fc492aeb4"
          }
        },
        "91105b770312483bae5338aab57359ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_863ab32db9024978ac3174c2656db292",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.68G/1.68G [00:23&lt;00:00, 72.9MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a057a4a9b5e54404b52060ebee6aae0a"
          }
        },
        "c4f2791a4fd84ca7a014c946349611d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "82c90ab4c7d649e097523d2fc492aeb4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "863ab32db9024978ac3174c2656db292": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a057a4a9b5e54404b52060ebee6aae0a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/agemagician/ProtTrans/blob/master/Embedding/PyTorch/Basic/ProtBert.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fXFGYhQo-h2F",
        "colab_type": "text"
      },
      "source": [
        "<h3> Extracting protein sequences' features using ProtBert pretrained-model <h3>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eRKqEJ5L-h2H",
        "colab_type": "text"
      },
      "source": [
        "<b>1. Load necessry libraries including huggingface transformers<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GXAKFATm-mbs",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "623498d6-bbc4-493d-d722-a0951c8441f9"
      },
      "source": [
        "!pip install -q transformers"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 778kB 8.5MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1MB 25.2MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.0MB 59.2MB/s \n",
            "\u001b[K     |████████████████████████████████| 890kB 55.1MB/s \n",
            "\u001b[?25h  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jd3YQUd1-h2I",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "from transformers import AutoTokenizer, AutoModel, pipeline\n",
        "import re\n",
        "import numpy as np\n",
        "import os\n",
        "import requests\n",
        "from tqdm.auto import tqdm"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hAKCMu_2-h2V",
        "colab_type": "text"
      },
      "source": [
        "<b>2. Load the vocabulary and ProtBert Model<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HS8i5sOJ-h2W",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 115,
          "referenced_widgets": [
            "c2f7806b750e4d3cbc7eca56b433e297",
            "552d3c4b5cc3492781115be450e4e3b7",
            "f6a30eb703924f39a2d1871cd5d790fa",
            "53daecb4f2d643ecb9dc87ba7b195318",
            "0e993b716be84e71806c8b0028522b2f",
            "1d103ef8e9a14ca5b1f57079372a9615",
            "aa3e0cd9d7d142ab976bc96c56ca295c",
            "f568e4e282254f98a46484d6cbf7e7f9",
            "36f263d60c1a4071a1ba3a4d3c156d47",
            "a935ca717870498cb1a524199005ce49",
            "142da5ae7175430f866e45c65f530499",
            "ba51cb0320d248b6b3213c00def9c2da",
            "5c79625a65164c6ab0b95a3f6eec2657",
            "704bf69a61354623851a86ac08214c6e",
            "671076a72248430d9829a73d1f282d7f",
            "0e46c95a421149c3890d08ac8370d6b2"
          ]
        },
        "outputId": "ee80e00b-5440-434c-c799-5c491e7a892a"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"Rostlab/prot_bert\", do_lower_case=False)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c2f7806b750e4d3cbc7eca56b433e297",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=313.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "36f263d60c1a4071a1ba3a4d3c156d47",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=81.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ERtkR05t-h2c",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "06f451a806284f0d90aeea93e347d009",
            "f3ce0768f9aa4ba3b5a1f6a25353c804",
            "bee98a37856642cd9b5e3c29672861c0",
            "91105b770312483bae5338aab57359ed",
            "c4f2791a4fd84ca7a014c946349611d6",
            "82c90ab4c7d649e097523d2fc492aeb4",
            "863ab32db9024978ac3174c2656db292",
            "a057a4a9b5e54404b52060ebee6aae0a"
          ]
        },
        "outputId": "cc258e28-8098-4e1e-d778-528361ab97cc"
      },
      "source": [
        "model = AutoModel.from_pretrained(\"Rostlab/prot_bert\")"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "06f451a806284f0d90aeea93e347d009",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1684058277.0, style=ProgressStyle(descr…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XDnhEiyI-h2g",
        "colab_type": "text"
      },
      "source": [
        "<b>3. Load the model into the GPU if avilabile<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KNweYMPQ-h2h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fe = pipeline('feature-extraction', model=model, tokenizer=tokenizer,device=0)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AWbZG3wu-h2l",
        "colab_type": "text"
      },
      "source": [
        "<b>4. Create or load sequences and map rarely occured amino acids (U,Z,O,B) to (X)<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2kqLvaxa-h2l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sequences_Example = [\"A E T C Z A O\",\"S K T Z P\"]"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rLxGKOwe-h2r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sequences_Example = [re.sub(r\"[UZOB]\", \"X\", sequence) for sequence in sequences_Example]"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jfk9vkyV-h2x",
        "colab_type": "text"
      },
      "source": [
        "<b>5. Extracting sequences' features and covert the output to numpy if needed<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gZPpLMK2-h2y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "embedding = fe(sequences_Example)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eRsQnB8Y-h24",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "embedding = np.array(embedding)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FIvh0j8e-h26",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 476
        },
        "outputId": "ae4efd7c-24eb-463d-97fc-430df621b39d"
      },
      "source": [
        "print(embedding)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[[ 0.04535879  0.11401558 -0.01174266 ... -0.08750971 -0.11434396\n",
            "    0.02040843]\n",
            "  [ 0.09233951  0.13914399 -0.05241546 ... -0.13947977 -0.0428006\n",
            "    0.07431364]\n",
            "  [ 0.11505969  0.01998261 -0.08627468 ... -0.00946233 -0.1872724\n",
            "    0.13169006]\n",
            "  ...\n",
            "  [ 0.10792391  0.09771117 -0.05832333 ... -0.12765658 -0.06493297\n",
            "    0.12894136]\n",
            "  [ 0.0545795   0.03635893 -0.07821195 ... -0.03016043 -0.06015311\n",
            "    0.08903892]\n",
            "  [ 0.05150128  0.0571134  -0.06932486 ... -0.03937564 -0.06630812\n",
            "    0.09767482]]\n",
            "\n",
            " [[ 0.06268476  0.08731148  0.00279815 ... -0.03485441 -0.037174\n",
            "    0.01748304]\n",
            "  [ 0.0880546   0.10885102  0.06212992 ...  0.0144938  -0.03327068\n",
            "   -0.0362049 ]\n",
            "  [-0.03332657  0.03442192  0.0809955  ...  0.0008738  -0.02733596\n",
            "    0.05783575]\n",
            "  ...\n",
            "  [ 0.00774996  0.03800717 -0.00171148 ... -0.00877388 -0.02430492\n",
            "    0.07620303]\n",
            "  [ 0.02335456  0.03041686 -0.00720565 ...  0.00245539 -0.01678618\n",
            "    0.09118143]\n",
            "  [ 0.03590559  0.02411232 -0.01498629 ...  0.00813052 -0.00361231\n",
            "    0.10301432]]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AuRYxkbO-h29",
        "colab_type": "text"
      },
      "source": [
        "<b>Optional: Remove padding ([PAD]) and special tokens ([CLS],[SEP]) that is added by Bert model<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HUL5nEBy-h2-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "features = [] \n",
        "\n",
        "for seq_num in range(len(embedding)):\n",
        "    seq_len = len(sequences_Example[seq_num].replace(\" \", \"\"))\n",
        "    start_Idx = 1\n",
        "    end_Idx = seq_len+1\n",
        "    seq_emd = embedding[seq_num][start_Idx:end_Idx]\n",
        "    features.append(seq_emd)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1-sP8qGp-h3B",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 391
        },
        "outputId": "b8b9153e-0bdc-4ef8-eec7-f94c9e7d1508"
      },
      "source": [
        "print(features)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[array([[ 0.09233951,  0.13914399, -0.05241546, ..., -0.13947977,\n",
            "        -0.0428006 ,  0.07431364],\n",
            "       [ 0.11505969,  0.01998261, -0.08627468, ..., -0.00946233,\n",
            "        -0.1872724 ,  0.13169006],\n",
            "       [ 0.04012022,  0.05467906, -0.0424584 , ...,  0.02054804,\n",
            "        -0.03819017,  0.1011811 ],\n",
            "       ...,\n",
            "       [ 0.08853199,  0.0228724 , -0.05121122, ..., -0.0673336 ,\n",
            "        -0.03013714,  0.04913732],\n",
            "       [ 0.10792391,  0.09771117, -0.05832333, ..., -0.12765658,\n",
            "        -0.06493297,  0.12894136],\n",
            "       [ 0.0545795 ,  0.03635893, -0.07821195, ..., -0.03016043,\n",
            "        -0.06015311,  0.08903892]]), array([[ 0.0880546 ,  0.10885102,  0.06212992, ...,  0.0144938 ,\n",
            "        -0.03327068, -0.0362049 ],\n",
            "       [-0.03332657,  0.03442192,  0.0809955 , ...,  0.0008738 ,\n",
            "        -0.02733596,  0.05783575],\n",
            "       [ 0.07896441,  0.10343076, -0.01508527, ...,  0.06711898,\n",
            "        -0.01733149,  0.12912118],\n",
            "       [ 0.01805276,  0.07072446, -0.03629201, ..., -0.03547218,\n",
            "         0.0481908 ,  0.0542736 ],\n",
            "       [ 0.05880854,  0.02880943, -0.08032861, ..., -0.00561737,\n",
            "         0.03985041,  0.10529047]])]\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}